import re
import requests
import time
#笔下文学666爬取小说
def search_text(html):
    req_html = requests.get(url=html)
    title_range=re.search('<div class="bookname">.*?<h1>(.*?)</h1>.*?</div>',req_html.text,re.S)
    Txt=list(title_range.groups())
    txt_find=re.findall('<div id="zjneirong">(.*?)<div>',req_html.text,re.S)
    Txt.append(re.sub('<p>','\n',txt_find[0]))
    return Txt

def next_section_html(html):
    req_html = requests.get(url=html)
    next_html=re.search('<a id="xiaye"  href="(.*?)">',req_html.text,re.S)
    return next_html.groups()[0]

def main():
    f=open('novel.txt','a',encoding='utf-8')
    URL='http://www.bxwx666.org/txt/251570/1113404.htm'
    num_sections=40
    for i in range(num_sections):
        text=search_text(URL)
        for i in text:
            f.write(i)
        time.sleep(10)
        URL=next_section_html(URL)
    f.close()
if __name__ == '__main__':
    main()
